[LV] Vectorize FMax w/o fast-math flags. #146711
Open
+566
−79
Conversation
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
llvmbot
added
vectorizers
llvm:analysis
|
@llvm/pr-subscribers-vectorizers @llvm/pr-subscribers-llvm-analysis Author: Florian Hahn (fhahn) ChangesAdd a new recurrence kind for FMax reductions without fast-math flags To do so, a new FindFirstIV reduction is added that tracks the first Patch is 99.08 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/146711.diff 19 Files Affected:
diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h
index 3b92cbff28de4..e1f0a5f6253ee 100644
--- a/llvm/include/llvm/Analysis/IVDescriptors.h
+++ b/llvm/include/llvm/Analysis/IVDescriptors.h
@@ -47,6 +47,8 @@ enum class RecurKind {
FMul, ///< Product of floats.
FMin, ///< FP min implemented in terms of select(cmp()).
FMax, ///< FP max implemented in terms of select(cmp()).
+ FMaxNoFMFs, ///< FP max implemented in terms of select(cmp()), but without
+ ///any fast-math flags. Users need to handle NaNs and signed zeros when generating code.
FMinimum, ///< FP min with llvm.minimum semantics
FMaximum, ///< FP max with llvm.maximum semantics
FMinimumNum, ///< FP min with llvm.minimumnum semantics
@@ -57,6 +59,9 @@ enum class RecurKind {
FindFirstIVSMin, /// FindFirst reduction with select(icmp(),x,y) where one of
///< (x,y) is a decreasing loop induction, and both x and y
///< are integer type, producing a SMin reduction.
+ FindFirstIVUMin, /// FindFirst reduction with select(icmp(),x,y) where one of
+ ///< (x,y) is a decreasing loop induction, and both x and y
+ ///< are integer type, producing a UMin reduction.
FindLastIVSMax, ///< FindLast reduction with select(cmp(),x,y) where one of
///< (x,y) is increasing loop induction, and both x and y
///< are integer type, producing a SMax reduction.
@@ -247,8 +252,9 @@ class RecurrenceDescriptor {
/// Returns true if the recurrence kind is a floating-point min/max kind.
static bool isFPMinMaxRecurrenceKind(RecurKind Kind) {
return Kind == RecurKind::FMin || Kind == RecurKind::FMax ||
- Kind == RecurKind::FMinimum || Kind == RecurKind::FMaximum ||
- Kind == RecurKind::FMinimumNum || Kind == RecurKind::FMaximumNum;
+ Kind == RecurKind::FMaxNoFMFs || Kind == RecurKind::FMinimum ||
+ Kind == RecurKind::FMaximum || Kind == RecurKind::FMinimumNum ||
+ Kind == RecurKind::FMaximumNum;
}
/// Returns true if the recurrence kind is any min/max kind.
@@ -265,7 +271,8 @@ class RecurrenceDescriptor {
/// Returns true if the recurrence kind is of the form
/// select(cmp(),x,y) where one of (x,y) is decreasing loop induction.
static bool isFindFirstIVRecurrenceKind(RecurKind Kind) {
- return Kind == RecurKind::FindFirstIVSMin;
+ return Kind == RecurKind::FindFirstIVSMin ||
+ Kind == RecurKind::FindFirstIVUMin;
}
/// Returns true if the recurrence kind is of the form
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index b275b1064cef2..6aad56f6291dc 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -51,6 +51,7 @@ bool RecurrenceDescriptor::isIntegerRecurrenceKind(RecurKind Kind) {
case RecurKind::UMin:
case RecurKind::AnyOf:
case RecurKind::FindFirstIVSMin:
+ case RecurKind::FindFirstIVUMin:
case RecurKind::FindLastIVSMax:
case RecurKind::FindLastIVUMax:
return true;
@@ -741,10 +742,9 @@ RecurrenceDescriptor::isFindIVPattern(RecurKind Kind, Loop *TheLoop,
: APInt::getMinValue(NumBits);
ValidRange = ConstantRange::getNonEmpty(Sentinel + 1, Sentinel);
} else {
- assert(IsSigned && "Only FindFirstIV with SMax is supported currently");
- ValidRange =
- ConstantRange::getNonEmpty(APInt::getSignedMinValue(NumBits),
- APInt::getSignedMaxValue(NumBits) - 1);
+ APInt Sentinel = IsSigned ? APInt::getSignedMaxValue(NumBits)
+ : APInt::getMaxValue(NumBits);
+ ValidRange = ConstantRange::getNonEmpty(Sentinel, Sentinel - 1);
}
LLVM_DEBUG(dbgs() << "LV: "
@@ -770,6 +770,8 @@ RecurrenceDescriptor::isFindIVPattern(RecurKind Kind, Loop *TheLoop,
if (CheckRange(true))
return RecurKind::FindFirstIVSMin;
+ if (CheckRange(false))
+ return RecurKind::FindFirstIVUMin;
return std::nullopt;
};
@@ -815,7 +817,8 @@ RecurrenceDescriptor::isMinMaxPattern(Instruction *I, RecurKind Kind,
if (match(I, m_OrdOrUnordFMin(m_Value(), m_Value())))
return InstDesc(Kind == RecurKind::FMin, I);
if (match(I, m_OrdOrUnordFMax(m_Value(), m_Value())))
- return InstDesc(Kind == RecurKind::FMax, I);
+ return InstDesc(Kind == RecurKind::FMax || Kind == RecurKind::FMaxNoFMFs,
+ I);
if (match(I, m_FMinNum(m_Value(), m_Value())))
return InstDesc(Kind == RecurKind::FMin, I);
if (match(I, m_FMaxNum(m_Value(), m_Value())))
@@ -937,10 +940,15 @@ RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(
m_Intrinsic<Intrinsic::minimumnum>(m_Value(), m_Value())) ||
match(I, m_Intrinsic<Intrinsic::maximumnum>(m_Value(), m_Value()));
};
- if (isIntMinMaxRecurrenceKind(Kind) ||
- (HasRequiredFMF() && isFPMinMaxRecurrenceKind(Kind)))
+ if (isIntMinMaxRecurrenceKind(Kind))
return isMinMaxPattern(I, Kind, Prev);
- else if (isFMulAddIntrinsic(I))
+ if (isFPMinMaxRecurrenceKind(Kind)) {
+ if (HasRequiredFMF())
+ return isMinMaxPattern(I, Kind, Prev);
+ if ((Kind == RecurKind::FMax || Kind == RecurKind::FMaxNoFMFs) &&
+ isMinMaxPattern(I, Kind, Prev).isRecurrence())
+ return InstDesc(I, RecurKind::FMaxNoFMFs);
+ } else if (isFMulAddIntrinsic(I))
return InstDesc(Kind == RecurKind::FMulAdd, I,
I->hasAllowReassoc() ? nullptr : I);
return InstDesc(false, I);
@@ -1183,6 +1191,7 @@ unsigned RecurrenceDescriptor::getOpcode(RecurKind Kind) {
return Instruction::Mul;
case RecurKind::AnyOf:
case RecurKind::FindFirstIVSMin:
+ case RecurKind::FindFirstIVUMin:
case RecurKind::FindLastIVSMax:
case RecurKind::FindLastIVUMax:
case RecurKind::Or:
@@ -1202,6 +1211,7 @@ unsigned RecurrenceDescriptor::getOpcode(RecurKind Kind) {
case RecurKind::UMin:
return Instruction::ICmp;
case RecurKind::FMax:
+ case RecurKind::FMaxNoFMFs:
case RecurKind::FMin:
case RecurKind::FMaximum:
case RecurKind::FMinimum:
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index e44fa6af29ffb..1074d9623a698 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -937,6 +937,7 @@ constexpr Intrinsic::ID llvm::getReductionIntrinsicID(RecurKind RK) {
return Intrinsic::vector_reduce_umax;
case RecurKind::UMin:
return Intrinsic::vector_reduce_umin;
+ case RecurKind::FMaxNoFMFs:
case RecurKind::FMax:
return Intrinsic::vector_reduce_fmax;
case RecurKind::FMin:
@@ -1085,6 +1086,7 @@ CmpInst::Predicate llvm::getMinMaxReductionPredicate(RecurKind RK) {
case RecurKind::FMin:
return CmpInst::FCMP_OLT;
case RecurKind::FMax:
+ case RecurKind::FMaxNoFMFs:
return CmpInst::FCMP_OGT;
// We do not add FMinimum/FMaximum recurrence kind here since there is no
// equivalent predicate which compares signed zeroes according to the
@@ -1307,6 +1309,7 @@ Value *llvm::createSimpleReduction(IRBuilderBase &Builder, Value *Src,
case RecurKind::UMax:
case RecurKind::UMin:
case RecurKind::FMax:
+ case RecurKind::FMaxNoFMFs:
case RecurKind::FMin:
case RecurKind::FMinimum:
case RecurKind::FMaximum:
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 907839711a39c..25182abe38a99 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4459,8 +4459,13 @@ bool LoopVectorizationPlanner::isCandidateForEpilogueVectorization(
ElementCount VF) const {
// Cross iteration phis such as reductions need special handling and are
// currently unsupported.
- if (any_of(OrigLoop->getHeader()->phis(),
- [&](PHINode &Phi) { return Legal->isFixedOrderRecurrence(&Phi); }))
+ if (any_of(OrigLoop->getHeader()->phis(), [&](PHINode &Phi) {
+ return Legal->isFixedOrderRecurrence(&Phi) ||
+ (Legal->isReductionVariable(&Phi) &&
+ Legal->getReductionVars()
+ .find(&Phi)
+ ->second.getRecurrenceKind() == RecurKind::FMaxNoFMFs);
+ }))
return false;
// Phis with uses outside of the loop require special handling and are
@@ -7267,8 +7272,7 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
auto *EpiRedHeaderPhi =
cast<VPReductionPHIRecipe>(EpiRedResult->getOperand(0));
- const RecurrenceDescriptor &RdxDesc =
- EpiRedHeaderPhi->getRecurrenceDescriptor();
+ RecurKind Kind = EpiRedHeaderPhi->getRecurrenceKind();
Value *MainResumeValue;
if (auto *VPI = dyn_cast<VPInstruction>(EpiRedHeaderPhi->getStartValue())) {
assert((VPI->getOpcode() == VPInstruction::Broadcast ||
@@ -7277,8 +7281,7 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
MainResumeValue = VPI->getOperand(0)->getUnderlyingValue();
} else
MainResumeValue = EpiRedHeaderPhi->getStartValue()->getUnderlyingValue();
- if (RecurrenceDescriptor::isAnyOfRecurrenceKind(
- RdxDesc.getRecurrenceKind())) {
+ if (RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind)) {
[[maybe_unused]] Value *StartV =
EpiRedResult->getOperand(1)->getLiveInIRValue();
auto *Cmp = cast<ICmpInst>(MainResumeValue);
@@ -7288,8 +7291,7 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
"AnyOf expected to start by comparing main resume value to original "
"start value");
MainResumeValue = Cmp->getOperand(0);
- } else if (RecurrenceDescriptor::isFindIVRecurrenceKind(
- RdxDesc.getRecurrenceKind())) {
+ } else if (RecurrenceDescriptor::isFindIVRecurrenceKind(Kind)) {
Value *StartV = getStartValueFromReductionResult(EpiRedResult);
Value *SentinelV = EpiRedResult->getOperand(2)->getLiveInIRValue();
using namespace llvm::PatternMatch;
@@ -8912,6 +8914,9 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
// Adjust the recipes for any inloop reductions.
adjustRecipesForReductions(Plan, RecipeBuilder, Range.Start);
+ if (!VPlanTransforms::runPass(
+ VPlanTransforms::handleFMaxReductionsWithoutFastMath, *Plan))
+ return nullptr;
// Transform recipes to abstract recipes if it is legal and beneficial and
// clamp the range for better cost estimation.
@@ -9070,8 +9075,7 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
if (!PhiR || !PhiR->isInLoop() || (MinVF.isScalar() && !PhiR->isOrdered()))
continue;
- const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor();
- RecurKind Kind = RdxDesc.getRecurrenceKind();
+ RecurKind Kind = PhiR->getRecurrenceKind();
assert(
!RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind) &&
!RecurrenceDescriptor::isFindIVRecurrenceKind(Kind) &&
@@ -9177,6 +9181,8 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
if (CM.blockNeedsPredicationForAnyReason(CurrentLinkI->getParent()))
CondOp = RecipeBuilder.getBlockInMask(CurrentLink->getParent());
+ const RecurrenceDescriptor &RdxDesc = Legal->getReductionVars().lookup(
+ cast<PHINode>(PhiR->getUnderlyingInstr()));
// Non-FP RdxDescs will have all fast math flags set, so clear them.
FastMathFlags FMFs = isa<FPMathOperator>(CurrentLinkI)
? RdxDesc.getFastMathFlags()
@@ -9207,7 +9213,8 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
if (!PhiR)
continue;
- const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor();
+ const RecurrenceDescriptor &RdxDesc = Legal->getReductionVars().lookup(
+ cast<PHINode>(PhiR->getUnderlyingInstr()));
Type *PhiTy = PhiR->getUnderlyingValue()->getType();
// If tail is folded by masking, introduce selects between the phi
// and the users outside the vector region of each reduction, at the
@@ -9853,14 +9860,9 @@ preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L,
}));
ResumeV = cast<PHINode>(ReductionPhi->getUnderlyingInstr())
->getIncomingValueForBlock(L->getLoopPreheader());
- const RecurrenceDescriptor &RdxDesc =
- ReductionPhi->getRecurrenceDescriptor();
- RecurKind RK = RdxDesc.getRecurrenceKind();
+ RecurKind RK = ReductionPhi->getRecurrenceKind();
if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RK)) {
Value *StartV = RdxResult->getOperand(1)->getLiveInIRValue();
- assert(RdxDesc.getRecurrenceStartValue() == StartV &&
- "start value from ComputeAnyOfResult must match");
-
// VPReductionPHIRecipes for AnyOf reductions expect a boolean as
// start value; compare the final value from the main vector loop
// to the start value.
@@ -9869,9 +9871,6 @@ preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L,
ResumeV = Builder.CreateICmpNE(ResumeV, StartV);
} else if (RecurrenceDescriptor::isFindIVRecurrenceKind(RK)) {
Value *StartV = getStartValueFromReductionResult(RdxResult);
- assert(RdxDesc.getRecurrenceStartValue() == StartV &&
- "start value from ComputeFinIVResult must match");
-
ToFrozen[StartV] = cast<PHINode>(ResumeV)->getIncomingValueForBlock(
EPI.MainLoopIterationCountCheck);
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 0941bf61953f1..54cc899df62b7 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -23181,6 +23181,7 @@ class HorizontalReduction {
case RecurKind::FMulAdd:
case RecurKind::AnyOf:
case RecurKind::FindFirstIVSMin:
+ case RecurKind::FindFirstIVUMin:
case RecurKind::FindLastIVSMax:
case RecurKind::FindLastIVUMax:
case RecurKind::FMaximumNum:
@@ -23317,6 +23318,7 @@ class HorizontalReduction {
case RecurKind::FMulAdd:
case RecurKind::AnyOf:
case RecurKind::FindFirstIVSMin:
+ case RecurKind::FindFirstIVUMin:
case RecurKind::FindLastIVSMax:
case RecurKind::FindLastIVUMax:
case RecurKind::FMaximumNum:
@@ -23418,8 +23420,10 @@ class HorizontalReduction {
case RecurKind::FMulAdd:
case RecurKind::AnyOf:
case RecurKind::FindFirstIVSMin:
+ case RecurKind::FindFirstIVUMin:
case RecurKind::FindLastIVSMax:
case RecurKind::FindLastIVUMax:
+ case RecurKind::FMaxNoFMFs:
case RecurKind::FMaximumNum:
case RecurKind::FMinimumNum:
case RecurKind::None:
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index d460573f5bec6..912a9d1b8a7e1 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -980,7 +980,10 @@ class VPInstruction : public VPRecipeWithIRFlags,
ReductionStartVector,
// Creates a step vector starting from 0 to VF with a step of 1.
StepVector,
-
+ /// Extracts a single lane (first operand) from a set of vector operands.
+ /// The lane specifies an index into a vector formed by combining all vector
+ /// operands (all operands after the first one).
+ ExtractLane,
};
private:
@@ -2192,7 +2195,7 @@ struct VPFirstOrderRecurrencePHIRecipe : public VPHeaderPHIRecipe {
class VPReductionPHIRecipe : public VPHeaderPHIRecipe,
public VPUnrollPartAccessor<2> {
/// Descriptor for the reduction.
- const RecurrenceDescriptor &RdxDesc;
+ const RecurKind Kind;
/// The phi is part of an in-loop reduction.
bool IsInLoop;
@@ -2211,16 +2214,24 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe,
VPValue &Start, bool IsInLoop = false,
bool IsOrdered = false, unsigned VFScaleFactor = 1)
: VPHeaderPHIRecipe(VPDef::VPReductionPHISC, Phi, &Start),
- RdxDesc(RdxDesc), IsInLoop(IsInLoop), IsOrdered(IsOrdered),
- VFScaleFactor(VFScaleFactor) {
+ Kind(RdxDesc.getRecurrenceKind()), IsInLoop(IsInLoop),
+ IsOrdered(IsOrdered), VFScaleFactor(VFScaleFactor) {
+ assert((!IsOrdered || IsInLoop) && "IsOrdered requires IsInLoop");
+ }
+ VPReductionPHIRecipe(PHINode *Phi, RecurKind Kind, VPValue &Start,
+ bool IsInLoop = false, bool IsOrdered = false,
+ unsigned VFScaleFactor = 1)
+ : VPHeaderPHIRecipe(VPDef::VPReductionPHISC, Phi, &Start), Kind(Kind),
+ IsInLoop(IsInLoop), IsOrdered(IsOrdered), VFScaleFactor(VFScaleFactor) {
assert((!IsOrdered || IsInLoop) && "IsOrdered requires IsInLoop");
}
~VPReductionPHIRecipe() override = default;
VPReductionPHIRecipe *clone() override {
+
auto *R = new VPReductionPHIRecipe(
- dyn_cast_or_null<PHINode>(getUnderlyingValue()), RdxDesc,
+ dyn_cast_or_null<PHINode>(getUnderlyingValue()), getRecurrenceKind(),
*getOperand(0), IsInLoop, IsOrdered, VFScaleFactor);
R->addOperand(getBackedgeValue());
return R;
@@ -2240,9 +2251,7 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe,
VPSlotTracker &SlotTracker) const override;
#endif
- const RecurrenceDescriptor &getRecurrenceDescriptor() const {
- return RdxDesc;
- }
+ RecurKind getRecurrenceKind() const { return Kind; }
/// Returns true, if the phi is part of an ordered reduction.
bool isOrdered() const { return IsOrdered; }
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 92db9674ef42b..1f802d60a2d36 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -85,6 +85,7 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
return ResTy;
}
case Instruction::ICmp:
+ case Instruction::FCmp:
case VPInstruction::ActiveLaneMask:
assert(inferScalarType(R->getOperand(0)) ==
inferScalarType(R->getOperand(1)) &&
@@ -110,6 +111,8 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
case VPInstruction::BuildStructVector:
case VPInstruction::BuildVector:
return SetResultTyFromOp();
+ case VPInstruction::ExtractLane:
+ return inferScalarType(R->getOperand(1));
case VPInstruction::FirstActiveLane:
return Type::getIntNTy(Ctx, 64);
case VPInstruction::ExtractLastElement:
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 593e5063802ba..168c605ab60ac 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -24,6 +24,7 @@
#define DEBUG_TYPE "vplan"
using namespace llvm;
+using namespace VPlanPatternMatch;
namespace {
// Class that is used to build the plain CFG for the incoming IR.
@@ -426,7 +427,6 @@ static void createLoopRegion(VPlan &Plan, VPBlockBase *HeaderVPB) {
static void addCanonicalIVRecipes(VPlan &Plan, VPBasicBlock *HeaderVPBB,
VPBasicBlock *LatchVPBB, Type *IdxTy,
DebugLoc DL) {
- using namespace VPlanPatternMatch;
Value *StartIdx = ConstantInt::get(IdxTy, 0);
auto *StartV = Plan.getOrAddLiveIn(StartIdx);
@@ -589,3 +589,114 @@ void VPlanTransforms::createLoopRegions(VPlan &Plan) {
TopRegion->setName("vector loop");
TopRegion->getEntryBasicBlock()->setName("vector.body");
}
+
+bool VPlanTransforms::handleFMaxReductionsWithoutFastMath(VPlan &Plan) {
+ VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
+ VPReductionPHIRecipe *RedPhiR = nullptr;
+ VPRecipeWithIRFlags *MinMaxOp = nullptr;
+ VPWidenIntOrFpInductionRecipe *WideIV = nullptr;
+
+ // Check if there are any FMaxNoFMFs reductions using wide selects that we can
+ // fix up. To do so, we also need a wide canonical IV to keep track of the
+ // indices of the max values.
+ for (auto &R : LoopRegion->getEntryBasicBlock()->phis()) {
+ // We need a wide canonical IV
+ if (auto *CurIV = dyn_cast<VPWidenIntOrFpInduct...
[truncated]
|
|
@llvm/pr-subscribers-llvm-transforms Author: Florian Hahn (fhahn) ChangesAdd a new recurrence kind for FMax reductions without fast-math flags To do so, a new FindFirstIV reduction is added that tracks the first Patch is 99.08 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/146711.diff 19 Files Affected:
diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h
index 3b92cbff28de4..e1f0a5f6253ee 100644
--- a/llvm/include/llvm/Analysis/IVDescriptors.h
+++ b/llvm/include/llvm/Analysis/IVDescriptors.h
@@ -47,6 +47,8 @@ enum class RecurKind {
FMul, ///< Product of floats.
FMin, ///< FP min implemented in terms of select(cmp()).
FMax, ///< FP max implemented in terms of select(cmp()).
+ FMaxNoFMFs, ///< FP max implemented in terms of select(cmp()), but without
+ ///any fast-math flags. Users need to handle NaNs and signed zeros when generating code.
FMinimum, ///< FP min with llvm.minimum semantics
FMaximum, ///< FP max with llvm.maximum semantics
FMinimumNum, ///< FP min with llvm.minimumnum semantics
@@ -57,6 +59,9 @@ enum class RecurKind {
FindFirstIVSMin, /// FindFirst reduction with select(icmp(),x,y) where one of
///< (x,y) is a decreasing loop induction, and both x and y
///< are integer type, producing a SMin reduction.
+ FindFirstIVUMin, /// FindFirst reduction with select(icmp(),x,y) where one of
+ ///< (x,y) is a decreasing loop induction, and both x and y
+ ///< are integer type, producing a UMin reduction.
FindLastIVSMax, ///< FindLast reduction with select(cmp(),x,y) where one of
///< (x,y) is increasing loop induction, and both x and y
///< are integer type, producing a SMax reduction.
@@ -247,8 +252,9 @@ class RecurrenceDescriptor {
/// Returns true if the recurrence kind is a floating-point min/max kind.
static bool isFPMinMaxRecurrenceKind(RecurKind Kind) {
return Kind == RecurKind::FMin || Kind == RecurKind::FMax ||
- Kind == RecurKind::FMinimum || Kind == RecurKind::FMaximum ||
- Kind == RecurKind::FMinimumNum || Kind == RecurKind::FMaximumNum;
+ Kind == RecurKind::FMaxNoFMFs || Kind == RecurKind::FMinimum ||
+ Kind == RecurKind::FMaximum || Kind == RecurKind::FMinimumNum ||
+ Kind == RecurKind::FMaximumNum;
}
/// Returns true if the recurrence kind is any min/max kind.
@@ -265,7 +271,8 @@ class RecurrenceDescriptor {
/// Returns true if the recurrence kind is of the form
/// select(cmp(),x,y) where one of (x,y) is decreasing loop induction.
static bool isFindFirstIVRecurrenceKind(RecurKind Kind) {
- return Kind == RecurKind::FindFirstIVSMin;
+ return Kind == RecurKind::FindFirstIVSMin ||
+ Kind == RecurKind::FindFirstIVUMin;
}
/// Returns true if the recurrence kind is of the form
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index b275b1064cef2..6aad56f6291dc 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -51,6 +51,7 @@ bool RecurrenceDescriptor::isIntegerRecurrenceKind(RecurKind Kind) {
case RecurKind::UMin:
case RecurKind::AnyOf:
case RecurKind::FindFirstIVSMin:
+ case RecurKind::FindFirstIVUMin:
case RecurKind::FindLastIVSMax:
case RecurKind::FindLastIVUMax:
return true;
@@ -741,10 +742,9 @@ RecurrenceDescriptor::isFindIVPattern(RecurKind Kind, Loop *TheLoop,
: APInt::getMinValue(NumBits);
ValidRange = ConstantRange::getNonEmpty(Sentinel + 1, Sentinel);
} else {
- assert(IsSigned && "Only FindFirstIV with SMax is supported currently");
- ValidRange =
- ConstantRange::getNonEmpty(APInt::getSignedMinValue(NumBits),
- APInt::getSignedMaxValue(NumBits) - 1);
+ APInt Sentinel = IsSigned ? APInt::getSignedMaxValue(NumBits)
+ : APInt::getMaxValue(NumBits);
+ ValidRange = ConstantRange::getNonEmpty(Sentinel, Sentinel - 1);
}
LLVM_DEBUG(dbgs() << "LV: "
@@ -770,6 +770,8 @@ RecurrenceDescriptor::isFindIVPattern(RecurKind Kind, Loop *TheLoop,
if (CheckRange(true))
return RecurKind::FindFirstIVSMin;
+ if (CheckRange(false))
+ return RecurKind::FindFirstIVUMin;
return std::nullopt;
};
@@ -815,7 +817,8 @@ RecurrenceDescriptor::isMinMaxPattern(Instruction *I, RecurKind Kind,
if (match(I, m_OrdOrUnordFMin(m_Value(), m_Value())))
return InstDesc(Kind == RecurKind::FMin, I);
if (match(I, m_OrdOrUnordFMax(m_Value(), m_Value())))
- return InstDesc(Kind == RecurKind::FMax, I);
+ return InstDesc(Kind == RecurKind::FMax || Kind == RecurKind::FMaxNoFMFs,
+ I);
if (match(I, m_FMinNum(m_Value(), m_Value())))
return InstDesc(Kind == RecurKind::FMin, I);
if (match(I, m_FMaxNum(m_Value(), m_Value())))
@@ -937,10 +940,15 @@ RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(
m_Intrinsic<Intrinsic::minimumnum>(m_Value(), m_Value())) ||
match(I, m_Intrinsic<Intrinsic::maximumnum>(m_Value(), m_Value()));
};
- if (isIntMinMaxRecurrenceKind(Kind) ||
- (HasRequiredFMF() && isFPMinMaxRecurrenceKind(Kind)))
+ if (isIntMinMaxRecurrenceKind(Kind))
return isMinMaxPattern(I, Kind, Prev);
- else if (isFMulAddIntrinsic(I))
+ if (isFPMinMaxRecurrenceKind(Kind)) {
+ if (HasRequiredFMF())
+ return isMinMaxPattern(I, Kind, Prev);
+ if ((Kind == RecurKind::FMax || Kind == RecurKind::FMaxNoFMFs) &&
+ isMinMaxPattern(I, Kind, Prev).isRecurrence())
+ return InstDesc(I, RecurKind::FMaxNoFMFs);
+ } else if (isFMulAddIntrinsic(I))
return InstDesc(Kind == RecurKind::FMulAdd, I,
I->hasAllowReassoc() ? nullptr : I);
return InstDesc(false, I);
@@ -1183,6 +1191,7 @@ unsigned RecurrenceDescriptor::getOpcode(RecurKind Kind) {
return Instruction::Mul;
case RecurKind::AnyOf:
case RecurKind::FindFirstIVSMin:
+ case RecurKind::FindFirstIVUMin:
case RecurKind::FindLastIVSMax:
case RecurKind::FindLastIVUMax:
case RecurKind::Or:
@@ -1202,6 +1211,7 @@ unsigned RecurrenceDescriptor::getOpcode(RecurKind Kind) {
case RecurKind::UMin:
return Instruction::ICmp;
case RecurKind::FMax:
+ case RecurKind::FMaxNoFMFs:
case RecurKind::FMin:
case RecurKind::FMaximum:
case RecurKind::FMinimum:
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index e44fa6af29ffb..1074d9623a698 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -937,6 +937,7 @@ constexpr Intrinsic::ID llvm::getReductionIntrinsicID(RecurKind RK) {
return Intrinsic::vector_reduce_umax;
case RecurKind::UMin:
return Intrinsic::vector_reduce_umin;
+ case RecurKind::FMaxNoFMFs:
case RecurKind::FMax:
return Intrinsic::vector_reduce_fmax;
case RecurKind::FMin:
@@ -1085,6 +1086,7 @@ CmpInst::Predicate llvm::getMinMaxReductionPredicate(RecurKind RK) {
case RecurKind::FMin:
return CmpInst::FCMP_OLT;
case RecurKind::FMax:
+ case RecurKind::FMaxNoFMFs:
return CmpInst::FCMP_OGT;
// We do not add FMinimum/FMaximum recurrence kind here since there is no
// equivalent predicate which compares signed zeroes according to the
@@ -1307,6 +1309,7 @@ Value *llvm::createSimpleReduction(IRBuilderBase &Builder, Value *Src,
case RecurKind::UMax:
case RecurKind::UMin:
case RecurKind::FMax:
+ case RecurKind::FMaxNoFMFs:
case RecurKind::FMin:
case RecurKind::FMinimum:
case RecurKind::FMaximum:
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 907839711a39c..25182abe38a99 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4459,8 +4459,13 @@ bool LoopVectorizationPlanner::isCandidateForEpilogueVectorization(
ElementCount VF) const {
// Cross iteration phis such as reductions need special handling and are
// currently unsupported.
- if (any_of(OrigLoop->getHeader()->phis(),
- [&](PHINode &Phi) { return Legal->isFixedOrderRecurrence(&Phi); }))
+ if (any_of(OrigLoop->getHeader()->phis(), [&](PHINode &Phi) {
+ return Legal->isFixedOrderRecurrence(&Phi) ||
+ (Legal->isReductionVariable(&Phi) &&
+ Legal->getReductionVars()
+ .find(&Phi)
+ ->second.getRecurrenceKind() == RecurKind::FMaxNoFMFs);
+ }))
return false;
// Phis with uses outside of the loop require special handling and are
@@ -7267,8 +7272,7 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
auto *EpiRedHeaderPhi =
cast<VPReductionPHIRecipe>(EpiRedResult->getOperand(0));
- const RecurrenceDescriptor &RdxDesc =
- EpiRedHeaderPhi->getRecurrenceDescriptor();
+ RecurKind Kind = EpiRedHeaderPhi->getRecurrenceKind();
Value *MainResumeValue;
if (auto *VPI = dyn_cast<VPInstruction>(EpiRedHeaderPhi->getStartValue())) {
assert((VPI->getOpcode() == VPInstruction::Broadcast ||
@@ -7277,8 +7281,7 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
MainResumeValue = VPI->getOperand(0)->getUnderlyingValue();
} else
MainResumeValue = EpiRedHeaderPhi->getStartValue()->getUnderlyingValue();
- if (RecurrenceDescriptor::isAnyOfRecurrenceKind(
- RdxDesc.getRecurrenceKind())) {
+ if (RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind)) {
[[maybe_unused]] Value *StartV =
EpiRedResult->getOperand(1)->getLiveInIRValue();
auto *Cmp = cast<ICmpInst>(MainResumeValue);
@@ -7288,8 +7291,7 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
"AnyOf expected to start by comparing main resume value to original "
"start value");
MainResumeValue = Cmp->getOperand(0);
- } else if (RecurrenceDescriptor::isFindIVRecurrenceKind(
- RdxDesc.getRecurrenceKind())) {
+ } else if (RecurrenceDescriptor::isFindIVRecurrenceKind(Kind)) {
Value *StartV = getStartValueFromReductionResult(EpiRedResult);
Value *SentinelV = EpiRedResult->getOperand(2)->getLiveInIRValue();
using namespace llvm::PatternMatch;
@@ -8912,6 +8914,9 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
// Adjust the recipes for any inloop reductions.
adjustRecipesForReductions(Plan, RecipeBuilder, Range.Start);
+ if (!VPlanTransforms::runPass(
+ VPlanTransforms::handleFMaxReductionsWithoutFastMath, *Plan))
+ return nullptr;
// Transform recipes to abstract recipes if it is legal and beneficial and
// clamp the range for better cost estimation.
@@ -9070,8 +9075,7 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
if (!PhiR || !PhiR->isInLoop() || (MinVF.isScalar() && !PhiR->isOrdered()))
continue;
- const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor();
- RecurKind Kind = RdxDesc.getRecurrenceKind();
+ RecurKind Kind = PhiR->getRecurrenceKind();
assert(
!RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind) &&
!RecurrenceDescriptor::isFindIVRecurrenceKind(Kind) &&
@@ -9177,6 +9181,8 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
if (CM.blockNeedsPredicationForAnyReason(CurrentLinkI->getParent()))
CondOp = RecipeBuilder.getBlockInMask(CurrentLink->getParent());
+ const RecurrenceDescriptor &RdxDesc = Legal->getReductionVars().lookup(
+ cast<PHINode>(PhiR->getUnderlyingInstr()));
// Non-FP RdxDescs will have all fast math flags set, so clear them.
FastMathFlags FMFs = isa<FPMathOperator>(CurrentLinkI)
? RdxDesc.getFastMathFlags()
@@ -9207,7 +9213,8 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
if (!PhiR)
continue;
- const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor();
+ const RecurrenceDescriptor &RdxDesc = Legal->getReductionVars().lookup(
+ cast<PHINode>(PhiR->getUnderlyingInstr()));
Type *PhiTy = PhiR->getUnderlyingValue()->getType();
// If tail is folded by masking, introduce selects between the phi
// and the users outside the vector region of each reduction, at the
@@ -9853,14 +9860,9 @@ preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L,
}));
ResumeV = cast<PHINode>(ReductionPhi->getUnderlyingInstr())
->getIncomingValueForBlock(L->getLoopPreheader());
- const RecurrenceDescriptor &RdxDesc =
- ReductionPhi->getRecurrenceDescriptor();
- RecurKind RK = RdxDesc.getRecurrenceKind();
+ RecurKind RK = ReductionPhi->getRecurrenceKind();
if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RK)) {
Value *StartV = RdxResult->getOperand(1)->getLiveInIRValue();
- assert(RdxDesc.getRecurrenceStartValue() == StartV &&
- "start value from ComputeAnyOfResult must match");
-
// VPReductionPHIRecipes for AnyOf reductions expect a boolean as
// start value; compare the final value from the main vector loop
// to the start value.
@@ -9869,9 +9871,6 @@ preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L,
ResumeV = Builder.CreateICmpNE(ResumeV, StartV);
} else if (RecurrenceDescriptor::isFindIVRecurrenceKind(RK)) {
Value *StartV = getStartValueFromReductionResult(RdxResult);
- assert(RdxDesc.getRecurrenceStartValue() == StartV &&
- "start value from ComputeFinIVResult must match");
-
ToFrozen[StartV] = cast<PHINode>(ResumeV)->getIncomingValueForBlock(
EPI.MainLoopIterationCountCheck);
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 0941bf61953f1..54cc899df62b7 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -23181,6 +23181,7 @@ class HorizontalReduction {
case RecurKind::FMulAdd:
case RecurKind::AnyOf:
case RecurKind::FindFirstIVSMin:
+ case RecurKind::FindFirstIVUMin:
case RecurKind::FindLastIVSMax:
case RecurKind::FindLastIVUMax:
case RecurKind::FMaximumNum:
@@ -23317,6 +23318,7 @@ class HorizontalReduction {
case RecurKind::FMulAdd:
case RecurKind::AnyOf:
case RecurKind::FindFirstIVSMin:
+ case RecurKind::FindFirstIVUMin:
case RecurKind::FindLastIVSMax:
case RecurKind::FindLastIVUMax:
case RecurKind::FMaximumNum:
@@ -23418,8 +23420,10 @@ class HorizontalReduction {
case RecurKind::FMulAdd:
case RecurKind::AnyOf:
case RecurKind::FindFirstIVSMin:
+ case RecurKind::FindFirstIVUMin:
case RecurKind::FindLastIVSMax:
case RecurKind::FindLastIVUMax:
+ case RecurKind::FMaxNoFMFs:
case RecurKind::FMaximumNum:
case RecurKind::FMinimumNum:
case RecurKind::None:
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index d460573f5bec6..912a9d1b8a7e1 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -980,7 +980,10 @@ class VPInstruction : public VPRecipeWithIRFlags,
ReductionStartVector,
// Creates a step vector starting from 0 to VF with a step of 1.
StepVector,
-
+ /// Extracts a single lane (first operand) from a set of vector operands.
+ /// The lane specifies an index into a vector formed by combining all vector
+ /// operands (all operands after the first one).
+ ExtractLane,
};
private:
@@ -2192,7 +2195,7 @@ struct VPFirstOrderRecurrencePHIRecipe : public VPHeaderPHIRecipe {
class VPReductionPHIRecipe : public VPHeaderPHIRecipe,
public VPUnrollPartAccessor<2> {
/// Descriptor for the reduction.
- const RecurrenceDescriptor &RdxDesc;
+ const RecurKind Kind;
/// The phi is part of an in-loop reduction.
bool IsInLoop;
@@ -2211,16 +2214,24 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe,
VPValue &Start, bool IsInLoop = false,
bool IsOrdered = false, unsigned VFScaleFactor = 1)
: VPHeaderPHIRecipe(VPDef::VPReductionPHISC, Phi, &Start),
- RdxDesc(RdxDesc), IsInLoop(IsInLoop), IsOrdered(IsOrdered),
- VFScaleFactor(VFScaleFactor) {
+ Kind(RdxDesc.getRecurrenceKind()), IsInLoop(IsInLoop),
+ IsOrdered(IsOrdered), VFScaleFactor(VFScaleFactor) {
+ assert((!IsOrdered || IsInLoop) && "IsOrdered requires IsInLoop");
+ }
+ VPReductionPHIRecipe(PHINode *Phi, RecurKind Kind, VPValue &Start,
+ bool IsInLoop = false, bool IsOrdered = false,
+ unsigned VFScaleFactor = 1)
+ : VPHeaderPHIRecipe(VPDef::VPReductionPHISC, Phi, &Start), Kind(Kind),
+ IsInLoop(IsInLoop), IsOrdered(IsOrdered), VFScaleFactor(VFScaleFactor) {
assert((!IsOrdered || IsInLoop) && "IsOrdered requires IsInLoop");
}
~VPReductionPHIRecipe() override = default;
VPReductionPHIRecipe *clone() override {
+
auto *R = new VPReductionPHIRecipe(
- dyn_cast_or_null<PHINode>(getUnderlyingValue()), RdxDesc,
+ dyn_cast_or_null<PHINode>(getUnderlyingValue()), getRecurrenceKind(),
*getOperand(0), IsInLoop, IsOrdered, VFScaleFactor);
R->addOperand(getBackedgeValue());
return R;
@@ -2240,9 +2251,7 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe,
VPSlotTracker &SlotTracker) const override;
#endif
- const RecurrenceDescriptor &getRecurrenceDescriptor() const {
- return RdxDesc;
- }
+ RecurKind getRecurrenceKind() const { return Kind; }
/// Returns true, if the phi is part of an ordered reduction.
bool isOrdered() const { return IsOrdered; }
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 92db9674ef42b..1f802d60a2d36 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -85,6 +85,7 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
return ResTy;
}
case Instruction::ICmp:
+ case Instruction::FCmp:
case VPInstruction::ActiveLaneMask:
assert(inferScalarType(R->getOperand(0)) ==
inferScalarType(R->getOperand(1)) &&
@@ -110,6 +111,8 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
case VPInstruction::BuildStructVector:
case VPInstruction::BuildVector:
return SetResultTyFromOp();
+ case VPInstruction::ExtractLane:
+ return inferScalarType(R->getOperand(1));
case VPInstruction::FirstActiveLane:
return Type::getIntNTy(Ctx, 64);
case VPInstruction::ExtractLastElement:
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 593e5063802ba..168c605ab60ac 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -24,6 +24,7 @@
#define DEBUG_TYPE "vplan"
using namespace llvm;
+using namespace VPlanPatternMatch;
namespace {
// Class that is used to build the plain CFG for the incoming IR.
@@ -426,7 +427,6 @@ static void createLoopRegion(VPlan &Plan, VPBlockBase *HeaderVPB) {
static void addCanonicalIVRecipes(VPlan &Plan, VPBasicBlock *HeaderVPBB,
VPBasicBlock *LatchVPBB, Type *IdxTy,
DebugLoc DL) {
- using namespace VPlanPatternMatch;
Value *StartIdx = ConstantInt::get(IdxTy, 0);
auto *StartV = Plan.getOrAddLiveIn(StartIdx);
@@ -589,3 +589,114 @@ void VPlanTransforms::createLoopRegions(VPlan &Plan) {
TopRegion->setName("vector loop");
TopRegion->getEntryBasicBlock()->setName("vector.body");
}
+
+bool VPlanTransforms::handleFMaxReductionsWithoutFastMath(VPlan &Plan) {
+ VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
+ VPReductionPHIRecipe *RedPhiR = nullptr;
+ VPRecipeWithIRFlags *MinMaxOp = nullptr;
+ VPWidenIntOrFpInductionRecipe *WideIV = nullptr;
+
+ // Check if there are any FMaxNoFMFs reductions using wide selects that we can
+ // fix up. To do so, we also need a wide canonical IV to keep track of the
+ // indices of the max values.
+ for (auto &R : LoopRegion->getEntryBasicBlock()->phis()) {
+ // We need a wide canonical IV
+ if (auto *CurIV = dyn_cast<VPWidenIntOrFpInduct...
[truncated]
|
arsenm
reviewed
Jul 7, 2025
Comment on lines
50
to
51
| FMaxNoFMFs, ///< FP max implemented in terms of select(cmp()), but without | ||
| ///any fast-math flags. Users need to handle NaNs and signed zeros when generating code. |
There was a problem hiding this comment.
Rename to something like FCmpOGTSelect?
There was a problem hiding this comment.
Yeah, I originally named if FMaxNoFMFs in line with existing FMax, but the current version doesn't match all pattersn FMax currently does. Renamed as suggested, we can always rename once/if we reach parity
There was a problem hiding this comment.
This captures a potentially Unordered FMax pattern - potentially with NaN's and signed zeroes - but not all possible such FMax patterns, yet? If all other cases should be added to this RecurKind, then that may sound like the right name? Perhaps the existing FMax pattern should be denoted as "Ordered", as it relies on FMF to indicate absence of NaN's (i.e., all elements are "isOrdered()") and signed zeroes.
Comment on lines
4465
to
4467
| Legal->getReductionVars() | ||
| .find(&Phi) | ||
| ->second.getRecurrenceKind() == RecurKind::FMaxNoFMFs); |
There was a problem hiding this comment.
.lookup? Can you shrink this by combining the isReductionVariable + getReductionVars.find?
There was a problem hiding this comment.
Updated to just use lookup, as it Kind will get default initialized to RecurKind::None
artagnon
reviewed
Jul 7, 2025
llvm/test/Transforms/LoopVectorize/ARM/mve-reduction-predselect.ll
Outdated
Show resolved
Hide resolved
Comment on lines
599
to
686
| // Check if there are any FMaxNoFMFs reductions using wide selects that we can | ||
| // fix up. To do so, we also need a wide canonical IV to keep track of the | ||
| // indices of the max values. | ||
| for (auto &R : LoopRegion->getEntryBasicBlock()->phis()) { | ||
| // We need a wide canonical IV | ||
| if (auto *CurIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&R)) { | ||
| if (!CurIV->isCanonical()) | ||
| continue; | ||
| WideIV = CurIV; | ||
| continue; | ||
| } | ||
|
|
||
| // And a single FMaxNoFMFs reduction phi. | ||
| // TODO: Support FMin reductions as well. | ||
| auto *CurRedPhiR = dyn_cast<VPReductionPHIRecipe>(&R); | ||
| if (!CurRedPhiR) | ||
| continue; | ||
| if (RedPhiR) | ||
| return false; | ||
| if (CurRedPhiR->getRecurrenceKind() != RecurKind::FMaxNoFMFs || | ||
| CurRedPhiR->isInLoop() || CurRedPhiR->isOrdered()) | ||
| continue; | ||
| RedPhiR = CurRedPhiR; | ||
|
|
||
| // MaxOp feeding the reduction phi must be a select (either wide or a | ||
| // replicate recipe), where the phi is the last operand, and the compare | ||
| // predicate is strict. This ensures NaNs won't get propagated unless the | ||
| // initial value is NaN | ||
| VPRecipeBase *Inc = RedPhiR->getBackedgeValue()->getDefiningRecipe(); | ||
| auto *RepR = dyn_cast<VPReplicateRecipe>(Inc); | ||
| if (!isa<VPWidenSelectRecipe>(Inc) && | ||
| !(RepR && (isa<SelectInst>(RepR->getUnderlyingInstr())))) | ||
| return false; | ||
|
|
||
| MinMaxOp = cast<VPRecipeWithIRFlags>(Inc); | ||
| auto *Cmp = cast<VPRecipeWithIRFlags>(MinMaxOp->getOperand(0)); | ||
| if (MinMaxOp->getOperand(1) == RedPhiR || | ||
| !CmpInst::isStrictPredicate(Cmp->getPredicate())) | ||
| return false; | ||
| } | ||
|
|
||
| // Nothing to do. | ||
| if (!RedPhiR) | ||
| return true; | ||
|
|
||
| // A wide canonical IV is currently required. | ||
| // TODO: Create an induction if no suitable existing one is available. | ||
| if (!WideIV) | ||
| return false; |
There was a problem hiding this comment.
Not sure why this can't be a function like isFindIVRecurrenceKind()?
There was a problem hiding this comment.
We need to check if there's a wide canonical induction, that is completely unconnected to the recurrence, so we would need to check all phi nodes. There are some other restrictions on the reductions that need to be checked to ensure the transform is legal. Even if we detect the full pattern early, we would still need to find the MaxOp and a sutable induction, so it would not save much work here.
Checking them directly when performing the transformation seems perferable and less fragile than checking in one place and applying the transform much later. It is also in line with general VPlan direction & initial design goal: reducing tight coupling between components by combining checks required for transforms directly in separate, de-coupled VPlan transforms.
Comment on lines
+8917
to
+8816
| if (!VPlanTransforms::runPass( | ||
| VPlanTransforms::handleFMaxReductionsWithoutFastMath, *Plan)) | ||
| return nullptr; |
There was a problem hiding this comment.
Not sure why the feature is designed to be in VPlanTransforms?
There was a problem hiding this comment.
llvm/lib/Analysis/IVDescriptors.cpp
Outdated
Comment on lines
948
to
950
| if ((Kind == RecurKind::FMax || Kind == RecurKind::FMaxNoFMFs) && | ||
| isMinMaxPattern(I, Kind, Prev).isRecurrence()) | ||
| return InstDesc(I, RecurKind::FMaxNoFMFs); |
There was a problem hiding this comment.
Doesn't this effectively remove the need for FMax? Can FMaxNoFMFs be folded into FMax? Ultimately, can we strip the hasRequiredFMFs() check once there are follow-ups handling No-FMFs for all the patterns?
There was a problem hiding this comment.
No not currently, as it would require checking the required FMFs in VPlan. Could be done eventually, but currently FMaxNoFMFs codegen does not support all FMax patterns
fhahn
force-pushed
the
lv-vectorize-fmax-without-fmfs
branch
3 times, most recently
from
06b1621 to
72b5b76
Compare
Add a new recurrence kind for FMax reductions without fast-math flags and a corresponding VPlan transform tries to vectorize without fast-math flags. To do so, a new FindFirstIV reduction is added that tracks the first indices that contain the maximum values. This serves as tie breaker if the partial reduction vector contains NaNs or signed zeros. After the loop, the first index is used to retrieve the final max value after vectorization from the vector containing the partial maximum values
fhahn
force-pushed
the
lv-vectorize-fmax-without-fmfs
branch
from
72b5b76 to
caae126
Compare
|
Ping. All dependencies have been merged now |
|
I also shared another variant to vectorize maxnum/minnum (used by std::fmax/std::fmin) in #148239. That one is slightly less code and may be good to review first |
ayalz
reviewed
Jul 12, 2025
| ; CHECK-NEXT: [[GEP_SRC1:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV1]] | ||
| ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC1]], i32 0 | ||
| ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4 | ||
| ; CHECK-NEXT: [[TMP2:%.*]] = fcmp ugt <4 x float> [[WIDE_LOAD]], [[VEC_PHI1]] |
There was a problem hiding this comment.
This seems to serve a FindLastIV rather than FindFirstIV, as the recorded IV is bumped every time the original fcmp ugt is true.
There was a problem hiding this comment.
FindFirstIV returns the lowest value in the reduction vector, ensuring that we pick the first induction value in the vector, irrespective of the predicate used to select the values of the vectors. Perhaps the naming could be improved, as regular FindFirstIV is only generated for reductions involving decrementing inductions.
| ; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] | ||
| ; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]] | ||
| ; CHECK: [[MIDDLE_BLOCK]]: | ||
| ; CHECK-NEXT: [[TMP11:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP8]]) |
There was a problem hiding this comment.
So TMP8 holds the correct partial reduction values, even in the presence of NaN's and signed zeroes (as when they are absent), issue is only how to finally reduce it to a single scalar here in the middle block?
TMP 11 holds the correct scalar value in the absence or NaNs and/or signed zeroes, but possibly an incorrect one in their presence? Would a serial or tree reduction of repeated binary operations provide the correct result in all cases (at least for fixed vectors) instead of calling llvm.vector.reduce.fmax()?
There was a problem hiding this comment.
So TMP8 holds the correct partial reduction values, even in the presence of NaN's and signed zeroes (as when they are absent), issue is only how to finally reduce it to a single scalar here in the middle block?
Yes, although we need to still pick the correct value in the presence of NaNs/Signed zeros.
TMP 11 holds the correct scalar value in the absence or NaNs and/or signed zeroes, but possibly an incorrect one in their presence? Would a serial or tree reduction of repeated binary operations provide the correct result in all cases (at least for fixed vectors) instead of calling llvm.vector.reduce.fmax()?
I don't think a serial tree reduction would work, as signed zeros are treated as equal, and we need to select the first encountered signed zero, if the result is zero. The signed zero encountered first does not have to be first in the partial reduction vector (e.g. could encounter -0.0 on iteration 1 and +0.0 on iteration 2, with VF 2 we would have <+0.0, -0.0> with index vector <2, 0>
Comment on lines
+489
to
+493
| ; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i64> poison, i64 [[RDX_SELECT]], i64 0 | ||
| ; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT3]], <4 x i64> poison, <4 x i32> zeroinitializer | ||
| ; CHECK-NEXT: [[TMP15:%.*]] = urem <4 x i64> [[BROADCAST_SPLAT4]], splat (i64 4) | ||
| ; CHECK-NEXT: [[TMP16:%.*]] = extractelement <4 x i64> [[TMP15]], i32 0 | ||
| ; CHECK-NEXT: [[TMP17:%.*]] = sub i64 [[TMP16]], 0 |
There was a problem hiding this comment.
Why broadcast the scalar RDX_SELECT, apply a vector urem by 4's, and extract first lane?
There was a problem hiding this comment.
Ah this looks like missed first lane codegen for VPInstruction
| ; CHECK-NEXT: [[TMP13:%.*]] = select <4 x i1> [[TMP12]], <4 x i64> [[TMP9]], <4 x i64> splat (i64 -1) | ||
| ; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vector.reduce.umin.v4i64(<4 x i64> [[TMP13]]) | ||
| ; CHECK-NEXT: [[RDX_SELECT_CMP:%.*]] = icmp ne i64 [[TMP14]], -1 | ||
| ; CHECK-NEXT: [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i64 [[TMP14]], i64 0 |
There was a problem hiding this comment.
-1 (max unsigned i64) which represents an out-of-bound index is replaced by zero which is a valid index for holding the desired result?
There was a problem hiding this comment.
Yep, in that case, the maximum has never been updated in the loop, selecting the first lane, which contains the initial value.
| ; CHECK-NEXT: [[TMP11:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP8]]) | ||
| ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP11]], i64 0 | ||
| ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer | ||
| ; CHECK-NEXT: [[TMP12:%.*]] = fcmp oeq <4 x float> [[TMP8]], [[BROADCAST_SPLAT]] |
There was a problem hiding this comment.
TMP12 indicates which lanes hold the value of TMP11 and are not a NaN?
| VPWidenIntOrFpInductionRecipe *WideIV = nullptr; | ||
|
|
||
| // Check if there are any FCmpOGTSelect reductions using wide selects that we | ||
| // can fix up. To do so, we also need a wide canonical IV to keep track of |
There was a problem hiding this comment.
Suggested change
| // can fix up. To do so, we also need a wide canonical IV to keep track of | |
| // can fix up. To do so, we also need a wide canonical IV to keep track of |
Comment on lines
+683
to
+686
| // A wide canonical IV is currently required. | ||
| // TODO: Create an induction if no suitable existing one is available. | ||
| if (!WideIV) | ||
| return false; |
There was a problem hiding this comment.
Note that a scalar canonical IV always exists, and is unique. But widen ones may exist (last one found is used?) or not.
There was a problem hiding this comment.
Yep, at this stage, all inductions will still be widened, but may not be canonical.
| MinIdxSel->insertAfter(MaxOp); | ||
| IdxPhi->addOperand(MinIdxSel); | ||
|
|
||
| // Find the first index of with the maximum value. This is used to extract the |
There was a problem hiding this comment.
Suggested change
| // Find the first index of with the maximum value. This is used to extract the | |
| // Find the first index holding the maximum value. This is used to extract the |
| {IdxPhi, WideIV->getStartValue(), UMinSentinel, IndicesWithMaxValue}); | ||
| // Convert the index of the first max value to an index in the vector lanes of | ||
| // the partial reduction results. This ensures we select the first max value | ||
| // and acts as a tie-breaker if the partial reductions contain signed zeros. |
There was a problem hiding this comment.
The vertical computation of each partial reduction result takes care of NaNs and signed zeroes, it is only the horizontal reduction of these vector lanes that require tie-breaking, to handle potential signed zeroes?
There was a problem hiding this comment.
Yes, the tie-breaking is only needed to handle signed zeroes when computing the final reduction results.
Consider a final partial reduction vector with -0.0, +0.0 and -0.0 was encountered before +0.0 (e.g. the max at iteration 2 is -0.0 and at iteration 3 it is +0.0. Doing a plain horizontal fmax reduction will produce +0.0 (-0.0 < +0.0).
We then compare the partial reduction values to the result of the horizontal reduction (-0.0 == +0.0 will also be true, selecting all lanes with zeros of any signed-ness)
Out of those, we select the one encountered first using FindFirstIV. Note that this only works for strict predicates.
Comment on lines
50
to
51
| FMaxNoFMFs, ///< FP max implemented in terms of select(cmp()), but without | ||
| ///any fast-math flags. Users need to handle NaNs and signed zeros when generating code. |
There was a problem hiding this comment.
This captures a potentially Unordered FMax pattern - potentially with NaN's and signed zeroes - but not all possible such FMax patterns, yet? If all other cases should be added to this RecurKind, then that may sound like the right name? Perhaps the existing FMax pattern should be denoted as "Ordered", as it relies on FMF to indicate absence of NaN's (i.e., all elements are "isOrdered()") and signed zeroes.
fhahn
added a commit
that referenced
this pull request
Jul 13, 2025
Adjust and update tests as per feedback in #146711.
llvm-sync bot
pushed a commit
to arm/arm-toolchain
that referenced
this pull request
Jul 13, 2025
Adjust and update tests as per feedback in llvm/llvm-project#146711.
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Add a new recurrence kind for FMax reductions without fast-math flags
and a corresponding VPlan transform tries to vectorize without fast-math
flags.
To do so, a new FindFirstIV reduction is added that tracks the first
indices that contain the maximum values. This serves as tie breaker
if the partial reduction vector contains NaNs or signed zeros.
After the loop, the first index is used to retrieve the final max
value after vectorization from the vector containing the partial
maximum values.
Depends on
#142322
#146386